import csv

import requests
from bs4 import BeautifulSoup


# 获取url中的所有内容
def getHtml(url):
    try:
        r = requests.get(url, timeout=30)
        r.encoding = 'utf-8'
        return r.text
    except:
        print("myError")


# 将html加入框架中
def createSoup(html):
    soup = BeautifulSoup(html, "html.parser")
    myValue = soup.find_all('tr')
    return myValue


all_values = []


def fillList(soup):
    for tr in soup:
        ltd = tr.find_all('td')
        if len(ltd) == 0:
            continue
        values = []
        for td in ltd:
            # print(td.string)
            # if td.string==None:
            #     for a in td:
            #         # print(a.string)
            #         if a.string==None:
            #             continue
            #         temp=a.string.replace("\n","").strip()
            #         # print(temp)
            #         if len(temp)>1:
            #             values.append(temp)
            if td != '\n':
                # print(td.text.strip())
                if len(td.text.strip()) > 30:
                    myText = td.text.strip().split(" ")[0]
                    # print(myText)
                    values.append(myText)
                else:
                    if len(td.text.strip()) >= 1:
                        # print(td.text.strip())
                        values.append(td.text.strip())
        # print(values)
        if len(values) > 0:
            all_values.append(values)


def createCsv():
    c=open("D:/PythonProject/20220426/pest_test/my.csv","w")
    writer=csv.writer(c)
    writer.writerows(all_values)
    c.close()

html = getHtml('https://shanghairanking.cn/rankings/bcur/2022')
data = createSoup(html)
fillList(data)
print(all_values)
createCsv()